package spider;

import com.google.common.collect.Lists;
import savantConn.mysql;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.http.Consts;
import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
//import org.jsoup.helper.StringUtil;
import org.jsoup.nodes.Document;

/**
 * 爬取知乎的话题信息
 */
public class topicSpider {

  public static Connection mConnect;
  public static Statement statement;

  public static List<Integer> listError = Lists.newArrayList();

  topicSpider() throws SQLException {

    mConnect = mysql.getConnection();
    statement = mConnect.createStatement();
  }

  /**
   * 获取主话题
   */
  public static void getTopicId() {
    new Thread(new Runnable() {

      @Override
      public void run() {
        try {
          String html = proxySpider.getIndex("https://www.zhihu.com/topics");

          String regex = "data-id=\"[0-9]{0,6}\"";
          Pattern pattern = Pattern.compile(regex);
          Matcher m = pattern.matcher(html);
          String regx = "href=\"#.*?\"";
          Pattern p = Pattern.compile(regx);
          Matcher mn = p.matcher(html);

          while (m.find() && mn.find()) {
            String topicId = m.group();
            topicId = topicId.substring(9, topicId.length() - 1);
            String topicName = mn.group();
            topicName = topicName.substring(7, topicName.length() - 1);
            System.out.println(topicId + " " + topicName);

//                        String sql ="INSERT INTO `topic`(`topicId`, `topicName`) VALUES ('"+topicId+"','"+topicName+"')";
//                        statement.execute(sql);//执行语句

            getChildTopics(Integer.parseInt(topicId));

          }

          System.out.println("----------------------------------------delimiter");
          System.out.println(listError);
          statement.close();
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        } finally {
        }
      }
    }).start();
  }

  /**
   * 获取主话题下的分话题
   *
   * @param parentId
   * @throws Exception
   */
  public static void getChildTopics(int parentId) throws Exception {

    int offset = 0;
    ;
    while (true) {
      PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
      CloseableHttpClient httpClient = HttpClients.custom()
          .setRetryHandler(new DefaultHttpRequestRetryHandler())
          .setConnectionManager(cm)
          .build();
      HttpPost req = new HttpPost("https://www.zhihu.com/node/TopicsPlazzaListV2");
      List<NameValuePair> params = new ArrayList<NameValuePair>();
      params.add(new BasicNameValuePair("method", "next"));
      params.add(new BasicNameValuePair("params",
          "{\"topic_id\":" + parentId + ",\"offset\":" + offset
              + ",\"hash_id\":\"37492588249aa9b50ee49d1797e9af81\"}"));
      req.setEntity(new UrlEncodedFormEntity(params, Consts.UTF_8));
      HttpResponse resp = httpClient.execute(req);
      String sb = EntityUtils.toString(resp.getEntity());
        if (sb.length() < 25) {
            break;
        }
      String regex = "<strong>.*?<\\\\/strong>";
      Pattern p = Pattern.compile(regex);
      Matcher m = p.matcher(sb);
      String regx = "href=\\\\\"\\\\/topic\\\\/[0-9]+\\\\\"";
      Pattern p1 = Pattern.compile(regx);
      Matcher m1 = p1.matcher(sb);

//            System.out.println(unicodeStr2String(sb));
      while (m.find() && m1.find()) {
        String topicid = m1.group().substring(16, m1.group().length() - 2);

        loadOneTopicHotQA(Integer.parseInt(topicid));

//                String temp = m.group().substring(8, m.group().length() - 10);
//                String topicName=unicodeStr2String(temp);
//
//                System.out.println(topicid + " " + topicName);
//                //转义单引号和双引号
//                topicName=topicName.replaceAll("'|\"","\\\\'");
//                String sql ="INSERT INTO `topic`(`topicId`, `topicName`, `parentId`) VALUES ('"+topicid+"','"+topicName+"','"+parentId+"')";
//                statement.execute(sql);//执行语句
      }
      offset += 20;
    }
  }


  public static void loadOne() throws Exception {
    String sql = "select topicId from topic where parentId!=0";
//        ResultSet rs=statement.executeQuery(sql);//执行语句
    List<Integer> list = Lists.newArrayList();

    StringBuilder sb = new StringBuilder();
    //        sb.append("");
    String str = sb.toString();

    for (String s : str.split(",")) {
      list.add(Integer.parseInt(s.trim()));
    }
    System.out.println(list.size());
//        while(rs.next()){
//            list.add(rs.getInt("topicId"));
//        }
//        rs.close();

    for (int i = 0; i < list.size(); i++) {
      System.out.print(i + " ");
      loadOneTopicHotQA(list.get(i));
    }
    System.out.println("----------------------------------------delimiter");
    System.out.println(listError);
    System.out.println(listError.size());
    statement.close();
  }

  /**
   * 通过分话题id来获取热门问答
   *
   * @param child_topic_id
   * @throws Exception
   */
  public static void loadOneTopicHotQA(int child_topic_id) throws Exception {

    PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();

    try {
      //超时问题
      RequestConfig config = RequestConfig.custom().setConnectTimeout(1000)
          .setSocketTimeout(2000).build();
      CloseableHttpClient httpClient = HttpClients.custom()
          .setRetryHandler(new DefaultHttpRequestRetryHandler())
          .setConnectionManager(cm)
          .setDefaultRequestConfig(config)
          .build();
      String result = "";
      HttpGet req = new HttpGet("https://www.zhihu.com/topic/" + child_topic_id + "/hot");
//        HttpGet req = new HttpGet("https://www.zhihu.com/topic/19560686/hot");

      //浏览器协议问题
      req.setProtocolVersion(HttpVersion.HTTP_1_0);
      HttpResponse resp = httpClient.execute(req);
      result = EntityUtils.toString(resp.getEntity());
      Document doc = Jsoup.parse(result);

      //获取话题名称
      String topicName = doc.select(
              "#root > div > main > div > div.ContentLayout-mainColumn > div.Card > div > div > div.ContentItem.TopicMetaCard-item > div > div.ContentItem-head > h2")
          .text();
      //话题关注者，问题数

//            Elements ele = doc.select("#root > div > main > div > div.ContentLayout-sideColumn > div > div > div > div > div");
//            Elements ele = doc.select("#root > div > main > div");

//        System.out.println(ele);
//            System.out.println(ele.text()+" "+child_topic_id);

//            String followers = ele.text().split(" ")[0].substring(3);
//        System.out.println(followers);
//            String questionNum = ele.text().split(" ")[1].substring(3);
//        System.out.println(questionNum);

      System.out.println(child_topic_id + " " + topicName);
      String sql = "INSERT INTO `topic2`(`topicId`, `followers`, `questionNum`) VALUES ('"
          + child_topic_id + "','" + "','" + "')";
      statement.execute(sql);//执行语句
    } catch (Exception e) {
      System.out.println(child_topic_id + " " + e);
      listError.add(child_topic_id);
//            throw new RuntimeException(e+ele.text());
    }

    //todo 爬取父话题，子话题，因为加载比较慢获取不到

    //todo 获取话题简介

//        Elements elements = doc.select("#TopicMain > div.ListShortcut > div > div > div > div");
    /**
     * 具体每一篇文章的发帖用户，用户链接，文章标题，点赞数，评论数
     * todo 还需要异步加载内容
     *
     */
//        for (Element element : elements) {
    /**
     * todo
     * 通过child_topic_id可知道问题属于哪个话题
     * 通过question_id可以通过拼接url:https://www.zhihu.com/question/{question_id}来访问问题
     * 同理可得url:https://www.zhihu.com/question/{question_id}/answer/{answer_id}来访问回答
     * 同理:https://www.zhihu.com/people/{username}/answers
     *
     */
//            #TopicMain > div.ListShortcut > div > div > div > div:nth-child(1) > div > div.ContentItem-meta > div > div
//            Elements e = element.select("div");
    //取文章标题
//            System.out.println(e.select(" h2").text());

    //取用户名称
//            System.out.println(e.select(" div.ContentItem-meta > div > div > div.AuthorInfo-head > span >a").text());

    //取用户链接 todo warn 判断取出来的是否空值
//            System.out.println(e.select("div.ContentItem-meta > div > div > div.AuthorInfo-head > span > a").attr("href").substring(2));

    //赞同数
//            System.out.println(e.select("div.RichContent.is-collapsed > div.ContentItem-actions > span > button.Button.VoteButton.VoteButton--up").attr("aria-label"));

    //评论数 todo warn 判断取出来的是否空值
//            System.out.println(e.select("div.RichContent.is-collapsed > div.ContentItem-actions > button").get(0).text());

//        }
  }

  /**
   * Unicode解码成字符串
   * @param src
   * @return
   */
  /**
   * 含有unicode 的字符串转一般字符串
   */
  public static String unicodeStr2String(String unicodeStr) {
//    if (StringUtil.isBlank(unicodeStr)) {
//      return "";
//    }
    int length = unicodeStr.length();
    int count = 0;
//        正则匹配条件，可匹配“\\u  1到4位，一般是4位可直接使用 ;
    String regex = "\\\\u[a-f0-9A-F]{1,4}";
    Pattern pattern = Pattern.compile(regex);
    Matcher matcher = pattern.matcher(unicodeStr);
    StringBuilder sb = new StringBuilder();

    while (matcher.find()) {
      String oldChar = matcher.group();//原本的Unicode字符
      String newChar = unicode2String(oldChar);//转换为普通字符
      // int index = unicodeStr.indexOf(oldChar);
      // 在遇见重复出现的unicode代码的时候会造成从源字符串获取非unicode编码字符的时候截取索引越界等
      int index = matcher.start();

      sb.append(unicodeStr, count, index);//添加前面不是unicode的字符
      sb.append(newChar);//添加转换后的字符
      count = index + oldChar.length();//统计下标移动的位置
    }
    sb.append(unicodeStr, count, length);//添加末尾不是Unicode的字符
    return sb.toString();
  }

  /**
   * unicode转字符串
   *
   * @param unicode
   * @return
   */
  private static String unicode2String(String unicode) {
    StringBuilder string = new StringBuilder();
    String[] hex = unicode.split("\\\\u");

    for (int i = 1; i < hex.length; i++) {
      // 转换出每一个代码点
      int data = Integer.parseInt(hex[i], 16);
      // 追加成string
      string.append((char) data);
    }
    return string.toString();
  }

}
